R Notebook

This is an R Markdown Notebook. When you execute code within the notebook, the results appear beneath the code.

Try executing this chunk by clicking the Run button within the chunk or by placing your cursor inside it and pressing Cmd+Shift+Enter.

# library for LMM 

library(lme4)
library(lmerTest)
library(car)

Loading required package: carData

df<-read.csv("overall_scores.csv", header =TRUE, sep=",")
df

mod <- lm( novelty ~ factor(Group), data = df)
summary(mod)

Call:
lm(formula = novelty ~ factor(Group), data = df)

Residuals:
    Min      1Q  Median      3Q     Max 
-38.636 -29.894  -6.789  33.033  80.893 

Coefficients:
               Estimate Std. Error t value Pr(>|t|)    
(Intercept)      25.357      2.440  10.391  < 2e-16 ***
factor(Group)1   13.279      3.389   3.918 9.67e-05 ***
factor(Group)2    6.432      3.389   1.898   0.0581 .  
factor(Group)3    4.537      3.409   1.331   0.1835    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 34.08 on 816 degrees of freedom
Multiple R-squared:  0.01925,   Adjusted R-squared:  0.01564 
F-statistic: 5.338 on 3 and 816 DF,  p-value: 0.001207

mod <- lm(user.requirement ~ factor(Group) , data = df)
summary(mod)

Call:
lm(formula = user.requirement ~ factor(Group), data = df)

Residuals:
   Min     1Q Median     3Q    Max 
-31.52 -20.51 -10.73  12.76  79.49 

Coefficients:
               Estimate Std. Error t value Pr(>|t|)    
(Intercept)      20.513      2.155   9.518  < 2e-16 ***
factor(Group)1   11.011      2.993   3.679 0.000249 ***
factor(Group)2    6.725      2.993   2.247 0.024902 *  
factor(Group)3   10.219      3.010   3.395 0.000721 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 30.09 on 816 degrees of freedom
Multiple R-squared:  0.01998,   Adjusted R-squared:  0.01637 
F-statistic: 5.545 on 3 and 816 DF,  p-value: 0.0009051

mod <- lm( infovis ~ factor(Group) , data = df)
summary(mod)

Call:
lm(formula = infovis ~ factor(Group), data = df)

Residuals:
    Min      1Q  Median      3Q     Max 
-38.631 -11.341  -3.011  13.659  59.402 

Coefficients:
               Estimate Std. Error t value Pr(>|t|)    
(Intercept)      28.098      1.502  18.711  < 2e-16 ***
factor(Group)1   10.533      2.085   5.051 5.44e-07 ***
factor(Group)2    5.969      2.085   2.862  0.00431 ** 
factor(Group)3    8.243      2.098   3.930 9.23e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 20.97 on 816 degrees of freedom
Multiple R-squared:  0.03314,   Adjusted R-squared:  0.02958 
F-statistic: 9.323 on 3 and 816 DF,  p-value: 4.589e-06

mod <- lm( total ~ factor(Group) , data = df)
summary(mod)

Call:
lm(formula = total ~ factor(Group), data = df)

Residuals:
    Min      1Q  Median      3Q     Max 
-165.34  -68.06  -18.11   74.71  235.56 

Coefficients:
               Estimate Std. Error t value Pr(>|t|)    
(Intercept)     117.365      6.973  16.832  < 2e-16 ***
factor(Group)1   47.973      9.683   4.954 8.84e-07 ***
factor(Group)2   29.826      9.683   3.080  0.00214 ** 
factor(Group)3   38.626      9.740   3.966 7.96e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 97.37 on 816 degrees of freedom
Multiple R-squared:  0.03233,   Adjusted R-squared:  0.02877 
F-statistic: 9.086 on 3 and 816 DF,  p-value: 6.39e-06

# convert to nominal factor
df$Group = factor(df$Group)
df$phase = factor(df$phase)

library(plyr)
ddply(df, ~ Group * phase, function(data) summary(data$novelty) )

ddply(df, ~ Group * phase, summarise, novelty.mean=mean(novelty), novelty.sd = sd(novelty))

# histograms for two factors
hist(df[df$Group == 0 & df$phase == 1,]$novelty)

hist(df[df$Group == 0 & df$phase == 2,]$novelty)

hist(df[df$Group == 0 & df$phase == 3,]$novelty)

hist(df[df$Group == 0 & df$phase == 4,]$novelty)

hist(df[df$Group == 1 & df$phase == 1,]$novelty)

hist(df[df$Group == 1 & df$phase == 2,]$novelty)

hist(df[df$Group == 1 & df$phase == 3,]$novelty)

hist(df[df$Group == 1 & df$phase == 4,]$novelty)

hist(df[df$Group == 2 & df$phase == 1,]$novelty)

hist(df[df$Group == 2 & df$phase == 2,]$novelty)

hist(df[df$Group == 2 & df$phase == 3,]$novelty)

hist(df[df$Group == 2 & df$phase == 4,]$novelty)

hist(df[df$Group == 3 & df$phase == 1,]$novelty)

hist(df[df$Group == 3 & df$phase == 2,]$novelty)

hist(df[df$Group == 3 & df$phase == 3,]$novelty)

hist(df[df$Group == 3 & df$phase == 4,]$novelty)

boxplot(novelty ~ Group * phase, data = df, xlab="Group.Phase", ylab="novelty")

with(df, interaction.plot(Group, phase, novelty, ylim=c(0, max(novelty)))) # interaction plot

m = lmer(novelty ~ Group + (1|Student), data=df, REML=FALSE)
summary(m)

Linear mixed model fit by maximum likelihood . t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: novelty ~ Group + (1 | Student)
   Data: df

     AIC      BIC   logLik deviance df.resid 
  7866.7   7894.9  -3927.3   7854.7      814 

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-2.5709 -0.5208 -0.1870  0.6010  3.0064 

Random effects:
 Groups   Name        Variance Std.Dev.
 Student  (Intercept) 545.0    23.34   
 Residual             602.1    24.54   
Number of obs: 820, groups:  Student, 163

Fixed effects:
            Estimate Std. Error      df t value Pr(>|t|)    
(Intercept)   25.357      4.131 163.852   6.139 6.05e-09 ***
Group1        12.230      5.766 163.429   2.121   0.0354 *  
Group2         6.432      5.736 163.852   1.121   0.2638    
Group3         4.537      5.770 163.852   0.786   0.4328    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Correlation of Fixed Effects:
       (Intr) Group1 Group2
Group1 -0.716              
Group2 -0.720  0.516       
Group3 -0.716  0.513  0.516

plot(resid(m, type = "pearson") ~ fitted(m))

qqnorm(resid(m, type = "pearson"))
qqline(resid(m, type = "pearson"))

# library for LMM we will use on relational novelty 

library(lme4)
library(lmerTest)
library(car)

set sum-to-zero contrast for ANOVA cells

contrasts(df$Group) <= "contr.sum"

     1    2    3
0 TRUE TRUE TRUE
1 TRUE TRUE TRUE
2 TRUE TRUE TRUE
3 TRUE TRUE TRUE

contrasts(df$phase) <= "contr.sum"

     2    3    4    5
1 TRUE TRUE TRUE TRUE
2 TRUE TRUE TRUE TRUE
3 TRUE TRUE TRUE TRUE
4 TRUE TRUE TRUE TRUE
5 TRUE TRUE TRUE TRUE

# phase is nested within group 
fit <- lmer(novelty ~ ( Group / phase ) +  ( 1 | Student), data = df, REML = FALSE)
Anova(fit, type=3, test.statistics="F")

Analysis of Deviance Table (Type III Wald chisquare tests)

Response: novelty
               Chisq Df Pr(>Chisq)    
(Intercept)   2.9277  1    0.08707 .  
Group         7.1045  3    0.06864 .  
Group:phase 118.9042 16    < 2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

fit

Linear mixed model fit by maximum likelihood  ['lmerModLmerTest']
Formula: novelty ~ (Group/phase) + (1 | Student)
   Data: df
      AIC       BIC    logLik  deviance  df.resid 
 7789.361  7892.966 -3872.681  7745.361       798 
Random effects:
 Groups   Name        Std.Dev.
 Student  (Intercept) 23.73   
 Residual             22.58   
Number of obs: 820, groups:  Student, 163
Fixed Effects:
  (Intercept)         Group1         Group2         Group3  Group0:phase2  Group1:phase2  Group2:phase2  Group3:phase2  Group0:phase3  
        8.974         17.914         15.728         10.893         13.141          7.609          7.440          3.150         18.109  
Group1:phase3  Group2:phase3  Group3:phase3  Group0:phase4  Group1:phase4  Group2:phase4  Group3:phase4  Group0:phase5  Group1:phase5  
       14.157          5.555          5.437         20.513         11.771          4.340         10.722         30.150         19.861  
Group2:phase5  Group3:phase5  
       18.095         30.823  

library(multcomp)

Loading required package: mvtnorm
Loading required package: survival
Loading required package: TH.data
Loading required package: MASS

Attaching package: ‘TH.data’

The following object is masked from ‘package:MASS’:

    geyser

library(lsmeans)

Loading required package: emmeans
The 'lsmeans' package is now basically a front end for 'emmeans'.
Users are encouraged to switch the rest of the way.
See help('transition') for more information, including how to
convert old 'lsmeans' objects and scripts to work with 'emmeans'.

summary(glht(fit, lsm(pairwise ~ Group / phase)), test = adjusted(type='holm'))

NOTE: A nesting structure was detected in the fitted model:
    phase %in% Group
Note: df set to 455

     Simultaneous Tests for General Linear Hypotheses

Fit: lmer(formula = novelty ~ (Group/phase) + (1 | Student), data = df, 
    REML = FALSE)

Linear Hypotheses:
                                   Estimate Std. Error t value Pr(>|t|)    
phase1 Group0 - phase2 Group0 == 0 -13.1410     5.1137  -2.570 1.000000    
phase1 Group0 - phase3 Group0 == 0 -18.1090     5.1137  -3.541 0.076425 .  
phase1 Group0 - phase4 Group0 == 0 -20.5128     5.1137  -4.011 0.012630 *  
phase1 Group0 - phase5 Group0 == 0 -30.1497     5.1137  -5.896 1.37e-06 ***
phase1 Group0 - phase1 Group1 == 0 -17.9138     7.3080  -2.451 1.000000    
phase1 Group0 - phase2 Group1 == 0 -25.5228     7.3080  -3.492 0.090896 .  
phase1 Group0 - phase3 Group1 == 0 -32.0704     7.3080  -4.388 0.002600 ** 
phase1 Group0 - phase4 Group1 == 0 -29.6845     7.3080  -4.062 0.010433 *  
phase1 Group0 - phase5 Group1 == 0 -37.7750     7.3080  -5.169 6.57e-05 ***
phase1 Group0 - phase1 Group2 == 0 -15.7280     7.2838  -2.159 1.000000    
phase1 Group0 - phase2 Group2 == 0 -23.1685     7.2838  -3.181 0.260457    
phase1 Group0 - phase3 Group2 == 0 -21.2835     7.2838  -2.922 0.584132    
phase1 Group0 - phase4 Group2 == 0 -20.0683     7.2838  -2.755 0.939603    
phase1 Group0 - phase5 Group2 == 0 -33.8235     7.2838  -4.644 0.000825 ***
phase1 Group0 - phase1 Group3 == 0 -10.8934     7.3265  -1.487 1.000000    
phase1 Group0 - phase2 Group3 == 0 -14.0439     7.3265  -1.917 1.000000    
phase1 Group0 - phase3 Group3 == 0 -16.3305     7.3265  -2.229 1.000000    
phase1 Group0 - phase4 Group3 == 0 -21.6152     7.3265  -2.950 0.540861    
phase1 Group0 - phase5 Group3 == 0 -41.7166     7.3265  -5.694 4.19e-06 ***
phase2 Group0 - phase3 Group0 == 0  -4.9679     5.1137  -0.972 1.000000    
phase2 Group0 - phase4 Group0 == 0  -7.3718     5.1137  -1.442 1.000000    
phase2 Group0 - phase5 Group0 == 0 -17.0087     5.1137  -3.326 0.162779    
phase2 Group0 - phase1 Group1 == 0  -4.7728     7.3080  -0.653 1.000000    
phase2 Group0 - phase2 Group1 == 0 -12.3818     7.3080  -1.694 1.000000    
phase2 Group0 - phase3 Group1 == 0 -18.9294     7.3080  -2.590 1.000000    
phase2 Group0 - phase4 Group1 == 0 -16.5435     7.3080  -2.264 1.000000    
phase2 Group0 - phase5 Group1 == 0 -24.6339     7.3080  -3.371 0.139903    
phase2 Group0 - phase1 Group2 == 0  -2.5870     7.2838  -0.355 1.000000    
phase2 Group0 - phase2 Group2 == 0 -10.0275     7.2838  -1.377 1.000000    
phase2 Group0 - phase3 Group2 == 0  -8.1425     7.2838  -1.118 1.000000    
phase2 Group0 - phase4 Group2 == 0  -6.9272     7.2838  -0.951 1.000000    
phase2 Group0 - phase5 Group2 == 0 -20.6825     7.2838  -2.840 0.736765    
phase2 Group0 - phase1 Group3 == 0   2.2476     7.3265   0.307 1.000000    
phase2 Group0 - phase2 Group3 == 0  -0.9029     7.3265  -0.123 1.000000    
phase2 Group0 - phase3 Group3 == 0  -3.1895     7.3265  -0.435 1.000000    
phase2 Group0 - phase4 Group3 == 0  -8.4741     7.3265  -1.157 1.000000    
phase2 Group0 - phase5 Group3 == 0 -28.5756     7.3265  -3.900 0.019679 *  
phase3 Group0 - phase4 Group0 == 0  -2.4038     5.1137  -0.470 1.000000    
phase3 Group0 - phase5 Group0 == 0 -12.0408     5.1137  -2.355 1.000000    
phase3 Group0 - phase1 Group1 == 0   0.1952     7.3080   0.027 1.000000    
phase3 Group0 - phase2 Group1 == 0  -7.4139     7.3080  -1.014 1.000000    
phase3 Group0 - phase3 Group1 == 0 -13.9615     7.3080  -1.910 1.000000    
phase3 Group0 - phase4 Group1 == 0 -11.5755     7.3080  -1.584 1.000000    
phase3 Group0 - phase5 Group1 == 0 -19.6660     7.3080  -2.691 1.000000    
phase3 Group0 - phase1 Group2 == 0   2.3810     7.2838   0.327 1.000000    
phase3 Group0 - phase2 Group2 == 0  -5.0595     7.2838  -0.695 1.000000    
phase3 Group0 - phase3 Group2 == 0  -3.1745     7.2838  -0.436 1.000000    
phase3 Group0 - phase4 Group2 == 0  -1.9593     7.2838  -0.269 1.000000    
phase3 Group0 - phase5 Group2 == 0 -15.7145     7.2838  -2.157 1.000000    
phase3 Group0 - phase1 Group3 == 0   7.2155     7.3265   0.985 1.000000    
phase3 Group0 - phase2 Group3 == 0   4.0650     7.3265   0.555 1.000000    
phase3 Group0 - phase3 Group3 == 0   1.7785     7.3265   0.243 1.000000    
phase3 Group0 - phase4 Group3 == 0  -3.5062     7.3265  -0.479 1.000000    
phase3 Group0 - phase5 Group3 == 0 -23.6076     7.3265  -3.222 0.229024    
phase4 Group0 - phase5 Group0 == 0  -9.6369     5.1137  -1.885 1.000000    
phase4 Group0 - phase1 Group1 == 0   2.5990     7.3080   0.356 1.000000    
phase4 Group0 - phase2 Group1 == 0  -5.0100     7.3080  -0.686 1.000000    
phase4 Group0 - phase3 Group1 == 0 -11.5576     7.3080  -1.581 1.000000    
phase4 Group0 - phase4 Group1 == 0  -9.1717     7.3080  -1.255 1.000000    
phase4 Group0 - phase5 Group1 == 0 -17.2621     7.3080  -2.362 1.000000    
phase4 Group0 - phase1 Group2 == 0   4.7848     7.2838   0.657 1.000000    
phase4 Group0 - phase2 Group2 == 0  -2.6557     7.2838  -0.365 1.000000    
phase4 Group0 - phase3 Group2 == 0  -0.7707     7.2838  -0.106 1.000000    
phase4 Group0 - phase4 Group2 == 0   0.4446     7.2838   0.061 1.000000    
phase4 Group0 - phase5 Group2 == 0 -13.3107     7.2838  -1.827 1.000000    
phase4 Group0 - phase1 Group3 == 0   9.6194     7.3265   1.313 1.000000    
phase4 Group0 - phase2 Group3 == 0   6.4689     7.3265   0.883 1.000000    
phase4 Group0 - phase3 Group3 == 0   4.1823     7.3265   0.571 1.000000    
phase4 Group0 - phase4 Group3 == 0  -1.1023     7.3265  -0.150 1.000000    
phase4 Group0 - phase5 Group3 == 0 -21.2038     7.3265  -2.894 0.633576    
phase5 Group0 - phase1 Group1 == 0  12.2360     7.3080   1.674 1.000000    
phase5 Group0 - phase2 Group1 == 0   4.6269     7.3080   0.633 1.000000    
phase5 Group0 - phase3 Group1 == 0  -1.9207     7.3080  -0.263 1.000000    
phase5 Group0 - phase4 Group1 == 0   0.4653     7.3080   0.064 1.000000    
phase5 Group0 - phase5 Group1 == 0  -7.6252     7.3080  -1.043 1.000000    
phase5 Group0 - phase1 Group2 == 0  14.4217     7.2838   1.980 1.000000    
phase5 Group0 - phase2 Group2 == 0   6.9812     7.2838   0.958 1.000000    
phase5 Group0 - phase3 Group2 == 0   8.8662     7.2838   1.217 1.000000    
phase5 Group0 - phase4 Group2 == 0  10.0815     7.2838   1.384 1.000000    
phase5 Group0 - phase5 Group2 == 0  -3.6738     7.2838  -0.504 1.000000    
phase5 Group0 - phase1 Group3 == 0  19.2563     7.3265   2.628 1.000000    
phase5 Group0 - phase2 Group3 == 0  16.1058     7.3265   2.198 1.000000    
phase5 Group0 - phase3 Group3 == 0  13.8192     7.3265   1.886 1.000000    
phase5 Group0 - phase4 Group3 == 0   8.5346     7.3265   1.165 1.000000    
phase5 Group0 - phase5 Group3 == 0 -11.5669     7.3265  -1.579 1.000000    
phase1 Group1 - phase2 Group1 == 0  -7.6090     4.9276  -1.544 1.000000    
phase1 Group1 - phase3 Group1 == 0 -14.1567     4.9276  -2.873 0.672665    
phase1 Group1 - phase4 Group1 == 0 -11.7707     4.9276  -2.389 1.000000    
phase1 Group1 - phase5 Group1 == 0 -19.8612     4.9276  -4.031 0.011805 *  
phase1 Group1 - phase1 Group2 == 0   2.1858     7.1723   0.305 1.000000    
phase1 Group1 - phase2 Group2 == 0  -5.2547     7.1723  -0.733 1.000000    
phase1 Group1 - phase3 Group2 == 0  -3.3697     7.1723  -0.470 1.000000    
phase1 Group1 - phase4 Group2 == 0  -2.1545     7.1723  -0.300 1.000000    
phase1 Group1 - phase5 Group2 == 0 -15.9097     7.1723  -2.218 1.000000    
phase1 Group1 - phase1 Group3 == 0   7.0203     7.2156   0.973 1.000000    
phase1 Group1 - phase2 Group3 == 0   3.8698     7.2156   0.536 1.000000    
phase1 Group1 - phase3 Group3 == 0   1.5833     7.2156   0.219 1.000000    
phase1 Group1 - phase4 Group3 == 0  -3.7014     7.2156  -0.513 1.000000    
phase1 Group1 - phase5 Group3 == 0 -23.8028     7.2156  -3.299 0.178036    
phase2 Group1 - phase3 Group1 == 0  -6.5476     4.9276  -1.329 1.000000    
phase2 Group1 - phase4 Group1 == 0  -4.1617     4.9276  -0.845 1.000000    
phase2 Group1 - phase5 Group1 == 0 -12.2521     4.9276  -2.486 1.000000    
phase2 Group1 - phase1 Group2 == 0   9.7948     7.1723   1.366 1.000000    
phase2 Group1 - phase2 Group2 == 0   2.3543     7.1723   0.328 1.000000    
phase2 Group1 - phase3 Group2 == 0   4.2393     7.1723   0.591 1.000000    
phase2 Group1 - phase4 Group2 == 0   5.4546     7.1723   0.760 1.000000    
phase2 Group1 - phase5 Group2 == 0  -8.3007     7.1723  -1.157 1.000000    
phase2 Group1 - phase1 Group3 == 0  14.6294     7.2156   2.027 1.000000    
phase2 Group1 - phase2 Group3 == 0  11.4789     7.2156   1.591 1.000000    
phase2 Group1 - phase3 Group3 == 0   9.1923     7.2156   1.274 1.000000    
phase2 Group1 - phase4 Group3 == 0   3.9077     7.2156   0.542 1.000000    
phase2 Group1 - phase5 Group3 == 0 -16.1938     7.2156  -2.244 1.000000    
phase3 Group1 - phase4 Group1 == 0   2.3860     4.9276   0.484 1.000000    
phase3 Group1 - phase5 Group1 == 0  -5.7045     4.9276  -1.158 1.000000    
phase3 Group1 - phase1 Group2 == 0  16.3424     7.1723   2.279 1.000000    
phase3 Group1 - phase2 Group2 == 0   8.9019     7.1723   1.241 1.000000    
phase3 Group1 - phase3 Group2 == 0  10.7869     7.1723   1.504 1.000000    
phase3 Group1 - phase4 Group2 == 0  12.0022     7.1723   1.673 1.000000    
phase3 Group1 - phase5 Group2 == 0  -1.7531     7.1723  -0.244 1.000000    
phase3 Group1 - phase1 Group3 == 0  21.1770     7.2156   2.935 0.564420    
phase3 Group1 - phase2 Group3 == 0  18.0265     7.2156   2.498 1.000000    
phase3 Group1 - phase3 Group3 == 0  15.7399     7.2156   2.181 1.000000    
phase3 Group1 - phase4 Group3 == 0  10.4553     7.2156   1.449 1.000000    
phase3 Group1 - phase5 Group3 == 0  -9.6462     7.2156  -1.337 1.000000    
phase4 Group1 - phase5 Group1 == 0  -8.0905     4.9276  -1.642 1.000000    
phase4 Group1 - phase1 Group2 == 0  13.9565     7.1723   1.946 1.000000    
phase4 Group1 - phase2 Group2 == 0   6.5160     7.1723   0.908 1.000000    
phase4 Group1 - phase3 Group2 == 0   8.4010     7.1723   1.171 1.000000    
phase4 Group1 - phase4 Group2 == 0   9.6162     7.1723   1.341 1.000000    
phase4 Group1 - phase5 Group2 == 0  -4.1390     7.1723  -0.577 1.000000    
phase4 Group1 - phase1 Group3 == 0  18.7910     7.2156   2.604 1.000000    
phase4 Group1 - phase2 Group3 == 0  15.6406     7.2156   2.168 1.000000    
phase4 Group1 - phase3 Group3 == 0  13.3540     7.2156   1.851 1.000000    
phase4 Group1 - phase4 Group3 == 0   8.0693     7.2156   1.118 1.000000    
phase4 Group1 - phase5 Group3 == 0 -12.0321     7.2156  -1.668 1.000000    
phase5 Group1 - phase1 Group2 == 0  22.0469     7.1723   3.074 0.369591    
phase5 Group1 - phase2 Group2 == 0  14.6065     7.1723   2.036 1.000000    
phase5 Group1 - phase3 Group2 == 0  16.4915     7.1723   2.299 1.000000    
phase5 Group1 - phase4 Group2 == 0  17.7067     7.1723   2.469 1.000000    
phase5 Group1 - phase5 Group2 == 0   3.9515     7.1723   0.551 1.000000    
phase5 Group1 - phase1 Group3 == 0  26.8815     7.2156   3.725 0.038848 *  
phase5 Group1 - phase2 Group3 == 0  23.7310     7.2156   3.289 0.183214    
phase5 Group1 - phase3 Group3 == 0  21.4444     7.2156   2.972 0.507933    
phase5 Group1 - phase4 Group3 == 0  16.1598     7.2156   2.240 1.000000    
phase5 Group1 - phase5 Group3 == 0  -3.9416     7.2156  -0.546 1.000000    
phase1 Group2 - phase2 Group2 == 0  -7.4405     4.9276  -1.510 1.000000    
phase1 Group2 - phase3 Group2 == 0  -5.5555     4.9276  -1.127 1.000000    
phase1 Group2 - phase4 Group2 == 0  -4.3402     4.9276  -0.881 1.000000    
phase1 Group2 - phase5 Group2 == 0 -18.0955     4.9276  -3.672 0.047345 *  
phase1 Group2 - phase1 Group3 == 0   4.8346     7.1911   0.672 1.000000    
phase1 Group2 - phase2 Group3 == 0   1.6841     7.1911   0.234 1.000000    
phase1 Group2 - phase3 Group3 == 0  -0.6025     7.1911  -0.084 1.000000    
phase1 Group2 - phase4 Group3 == 0  -5.8871     7.1911  -0.819 1.000000    
phase1 Group2 - phase5 Group3 == 0 -25.9886     7.1911  -3.614 0.058656 .  
phase2 Group2 - phase3 Group2 == 0   1.8850     4.9276   0.383 1.000000    
phase2 Group2 - phase4 Group2 == 0   3.1002     4.9276   0.629 1.000000    
phase2 Group2 - phase5 Group2 == 0 -10.6550     4.9276  -2.162 1.000000    
phase2 Group2 - phase1 Group3 == 0  12.2751     7.1911   1.707 1.000000    
phase2 Group2 - phase2 Group3 == 0   9.1246     7.1911   1.269 1.000000    
phase2 Group2 - phase3 Group3 == 0   6.8380     7.1911   0.951 1.000000    
phase2 Group2 - phase4 Group3 == 0   1.5533     7.1911   0.216 1.000000    
phase2 Group2 - phase5 Group3 == 0 -18.5481     7.1911  -2.579 1.000000    
phase3 Group2 - phase4 Group2 == 0   1.2152     4.9276   0.247 1.000000    
phase3 Group2 - phase5 Group2 == 0 -12.5400     4.9276  -2.545 1.000000    
phase3 Group2 - phase1 Group3 == 0  10.3901     7.1911   1.445 1.000000    
phase3 Group2 - phase2 Group3 == 0   7.2396     7.1911   1.007 1.000000    
phase3 Group2 - phase3 Group3 == 0   4.9530     7.1911   0.689 1.000000    
phase3 Group2 - phase4 Group3 == 0  -0.3317     7.1911  -0.046 1.000000    
phase3 Group2 - phase5 Group3 == 0 -20.4331     7.1911  -2.841 0.736765    
phase4 Group2 - phase5 Group2 == 0 -13.7552     4.9276  -2.791 0.847574    
phase4 Group2 - phase1 Group3 == 0   9.1748     7.1911   1.276 1.000000    
phase4 Group2 - phase2 Group3 == 0   6.0243     7.1911   0.838 1.000000    
phase4 Group2 - phase3 Group3 == 0   3.7377     7.1911   0.520 1.000000    
phase4 Group2 - phase4 Group3 == 0  -1.5469     7.1911  -0.215 1.000000    
phase4 Group2 - phase5 Group3 == 0 -21.6484     7.1911  -3.010 0.451634    
phase5 Group2 - phase1 Group3 == 0  22.9301     7.1911   3.189 0.255168    
phase5 Group2 - phase2 Group3 == 0  19.7796     7.1911   2.751 0.946564    
phase5 Group2 - phase3 Group3 == 0  17.4930     7.1911   2.433 1.000000    
phase5 Group2 - phase4 Group3 == 0  12.2083     7.1911   1.698 1.000000    
phase5 Group2 - phase5 Group3 == 0  -7.8931     7.1911  -1.098 1.000000    
phase1 Group3 - phase2 Group3 == 0  -3.1505     4.9874  -0.632 1.000000    
phase1 Group3 - phase3 Group3 == 0  -5.4371     4.9874  -1.090 1.000000    
phase1 Group3 - phase4 Group3 == 0 -10.7217     4.9874  -2.150 1.000000    
phase1 Group3 - phase5 Group3 == 0 -30.8232     4.9874  -6.180 2.70e-07 ***
phase2 Group3 - phase3 Group3 == 0  -2.2866     4.9874  -0.458 1.000000    
phase2 Group3 - phase4 Group3 == 0  -7.5712     4.9874  -1.518 1.000000    
phase2 Group3 - phase5 Group3 == 0 -27.6727     4.9874  -5.549 9.16e-06 ***
phase3 Group3 - phase4 Group3 == 0  -5.2846     4.9874  -1.060 1.000000    
phase3 Group3 - phase5 Group3 == 0 -25.3861     4.9874  -5.090 9.71e-05 ***
phase4 Group3 - phase5 Group3 == 0 -20.1015     4.9874  -4.030 0.011805 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
(Adjusted p values reported -- holm method)

fit.full <- lmer(novelty ~ ( Group / phase ) + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + ( 1 | Student), data = df, REML = FALSE)
Anova(fit.full, type=3, test.statistics="F")

Analysis of Deviance Table (Type III Wald chisquare tests)

Response: novelty
               Chisq Df Pr(>Chisq)    
(Intercept)   1.0102  1   0.314852    
Group         6.5217  3   0.088811 .  
Q7_Q7_1       9.2241  1   0.002388 ** 
Q7_Q7_2       5.0628  1   0.024445 *  
Q8_Q8_1       1.0258  1   0.311156    
Q10           4.4038  1   0.035860 *  
Group:phase 117.2473 16  < 2.2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

fit.full

Linear mixed model fit by maximum likelihood  ['lmerModLmerTest']
Formula: novelty ~ (Group/phase) + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 +      (1 | Student)
   Data: df
      AIC       BIC    logLik  deviance  df.resid 
 7598.296  7720.096 -3773.148  7546.296       774 
Random effects:
 Groups   Name        Std.Dev.
 Student  (Intercept) 22.25   
 Residual             22.70   
Number of obs: 800, groups:  Student, 159
Fixed Effects:
  (Intercept)         Group1         Group2         Group3        Q7_Q7_1        Q7_Q7_2        Q8_Q8_1            Q10  Group0:phase2  
       -8.570         16.857         15.288         10.127         -4.858          3.739          1.701          5.196         13.851  
Group1:phase2  Group2:phase2  Group3:phase2  Group0:phase3  Group1:phase3  Group2:phase3  Group3:phase3  Group0:phase4  Group1:phase4  
        7.795          6.402          3.150         19.088         14.502          5.691          5.437         21.622         12.058  
Group2:phase4  Group3:phase4  Group0:phase5  Group1:phase5  Group2:phase5  Group3:phase5  
        4.446         10.722         30.158         20.346         18.537         30.823  

User Requirement Score

# phase is nested within group 
fit.requirement.full <- lmer(user.requirement ~ ( Group / phase ) + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 + Q10 + ( 1 | Student), data = df, REML = FALSE)
Anova(fit.requirement.full, type=3, test.statistics="F")

Analysis of Deviance Table (Type III Wald chisquare tests)

Response: user.requirement
               Chisq Df Pr(>Chisq)    
(Intercept)   1.1511  1    0.28331    
Group         2.6216  3    0.45371    
Q7_Q7_1       6.1467  1    0.01317 *  
Q7_Q7_2       3.2341  1    0.07212 .  
Q8_Q8_1       0.9493  1    0.32990    
Q10           5.4933  1    0.01909 *  
Group:phase 164.9608 16    < 2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

fit.requirement.full

Linear mixed model fit by maximum likelihood  ['lmerModLmerTest']
Formula: user.requirement ~ (Group/phase) + Q7_Q7_1 + Q7_Q7_2 + Q8_Q8_1 +      Q10 + (1 | Student)
   Data: df
      AIC       BIC    logLik  deviance  df.resid 
 7335.734  7457.534 -3641.867  7283.734       774 
Random effects:
 Groups   Name        Std.Dev.
 Student  (Intercept) 20.70   
 Residual             18.91   
Number of obs: 800, groups:  Student, 159
Fixed Effects:
  (Intercept)         Group1         Group2         Group3        Q7_Q7_1        Q7_Q7_2        Q8_Q8_1            Q10  Group0:phase2  
       -8.258          9.712          5.259          8.142         -3.623          2.733          1.497          5.308          4.324  
Group1:phase2  Group2:phase2  Group3:phase2  Group0:phase3  Group1:phase3  Group2:phase3  Group3:phase3  Group0:phase4  Group1:phase4  
        4.390          7.317         10.732         16.757         13.171         15.610         12.195         14.054         17.073  
Group2:phase4  Group3:phase4  Group0:phase5  Group1:phase5  Group2:phase5  Group3:phase5  
       19.512         16.098         24.324         19.512         23.902         29.268  

# histograms for two factors
boxplot(user.requirement ~ Group * phase, data = df, xlab="Group.Phase", ylab="user.requirement")

with(df, interaction.plot(Group, phase, user.requirement, ylim=c(0, max(user.requirement)))) # interaction plot

# phase is nested within group 
fit.requirement <- lmer(user.requirement ~ ( Group / phase ) +  ( 1 | Student), data = df, REML = FALSE)
Anova(fit, type=3, test.statistics="F")

Analysis of Deviance Table (Type III Wald chisquare tests)

Response: novelty
               Chisq Df Pr(>Chisq)    
(Intercept)   2.9277  1    0.08707 .  
Group         7.1045  3    0.06864 .  
Group:phase 118.9042 16    < 2e-16 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

fit.requirement

Linear mixed model fit by maximum likelihood  ['lmerModLmerTest']
Formula: user.requirement ~ (Group/phase) + (1 | Student)
   Data: df
      AIC       BIC    logLik  deviance  df.resid 
 7513.970  7617.574 -3734.985  7469.970       798 
Random effects:
 Groups   Name        Std.Dev.
 Student  (Intercept) 21.6    
 Residual             18.8    
Number of obs: 820, groups:  Student, 163
Fixed Effects:
  (Intercept)         Group1         Group2         Group3  Group0:phase2  Group1:phase2  Group2:phase2  Group3:phase2  Group0:phase3  
        8.718         11.137          5.568          8.355          4.615          4.286          7.143         10.732         15.897  
Group1:phase3  Group2:phase3  Group3:phase3  Group0:phase4  Group1:phase4  Group2:phase4  Group3:phase4  Group0:phase5  Group1:phase5  
       12.857         15.238         12.195         14.359         16.667         19.048         16.098         24.103         19.048  
Group2:phase5  Group3:phase5  
       23.333         29.268  

plot(resid(m, type = "pearson") ~ fitted(m))
qqnorm(resid(m, type = "pearson"))
qqline(resid(m, type = "pearson"))

anova(fit.requirement, fit.requirement.full)

Error in anova.merMod(fit.requirement, fit.requirement.full) : 
  models were not all fitted to the same size of dataset